package com.guiji.quartz.task;

import com.guiji.quartz.domain.Gadgets;
import com.guiji.quartz.domain.Techdata;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.downloader.HttpClientDownloader;
import us.codecraft.webmagic.model.ConsolePageModelPipeline;
import us.codecraft.webmagic.model.OOSpider;
import us.codecraft.webmagic.proxy.Proxy;
import us.codecraft.webmagic.proxy.SimpleProxyProvider;

/**
 * @program: cms-vue-plus
 * @description:
 * @author: gaoX
 * @create: 2021-10-13 17:11
 **/
@Component("techdataTask")
public class TechdataTask {
	private static final Logger log = LoggerFactory.getLogger(TechdataTask.class);

	public void techdataTask() throws InterruptedException {
		Long startTime = System.currentTimeMillis();
		Spider spider = OOSpider.create(Site.me().setSleepTime(1000), new ConsolePageModelPipeline(), Techdata.class)
			.addUrl("https://techdatanewsflash.co.uk/news")
			.thread(20);
		spider.run();

		log.info("techdataTask-end-:{}", System.currentTimeMillis() - startTime);
	}
}
